import sys
import time
import aiohttp
import asyncio
import requests
from queue import Queue
from random import uniform
from Spider.core.UrlManage import FifoQueue
from Spider.core.HandleMiddleware import Handle
from Spider.Exception.exceptions import SettingError, ObjectTypeError
from Spider.util.MyFuncClass import HttpRequest, HttpResponse, add_proxy
from Spider.settings import PROXY_TYPE, REQUEST_MAX_NUMBER, DELAY_ENABLED,DELAY_TIME, TUNNEL_PROXY_CONFIG


class Downloader(object):
    def __init__(self):
        self.queue = FifoQueue()
        self.success_response_queue = Queue()
        self.loop = asyncio.get_event_loop()
        self.middleware = Handle()
        self.success_count = 0      # 下载成功页数

    async def async_download(self, request):
        async with aiohttp.ClientSession() as session:
            try:
                async with await session.get(url=request.url, 
                                             headers=request.headers,
                                             cookies=request.cookies,
                                             proxy=request.proxy,
                                             timeout=request.timeout) as response:
                    text = await response.text()   # json类型的字符串
                    obj = self.middleware.response_ware(request=request, response=HttpResponse(status=response.status,
                                                                                               text_data=text,
                                                                                               callback=request.callback,
                                                                                               meta=request.meta))
                    if isinstance(obj, HttpRequest):
                        self.queue.push(request)
                        self.success_response_queue.put(None)
                    elif isinstance(obj, HttpResponse):
                        self.success_response_queue.put(obj)
                        self.success_count += 1
                        print('下载页面成功总数:', self.success_count)
            # 请求超时，连接失败，代理不可用
            except (aiohttp.ClientConnectionError, aiohttp.ClientError, asyncio.TimeoutError, aiohttp.ClientHttpProxyError):
                city = request.meta.get('city')
                page = request.meta.get('page')
                fail_time = request.fail_time
                print(f'城市:{city}  页码:{page}  失败次数:{fail_time}  请求发生异常，请求对象的失败次数+1')
                self.queue.push(request)
                self.success_response_queue.put(None)

    def run_downloader(self):
        objs = []
        while True:
            obj = self.queue.pop()
            if obj:
                obj = self.middleware.request_ware(obj)
                objs.append(obj)
            else:
                # 没有取到obj。 1.列表中有1或2个obj   2.列表中没有obj
                # 列表中没有obj
                if len(objs) == 0:
                    break
                # 列表中有1或2个obj
                self.create_task(objs=objs)
                break
            if len(objs) == REQUEST_MAX_NUMBER:
                self.create_task(objs=objs)
                objs = []
                if DELAY_ENABLED:
                    time.sleep(uniform(DELAY_TIME, DELAY_TIME + 1))
        print('所有页面下载完成')

    @add_proxy
    def create_task(self, objs):
        if isinstance(objs, list):
            tasks = [self.async_download(obj) for obj in objs]
            self.loop.run_until_complete(asyncio.wait(tasks))

    def get_proxy(self):
        if PROXY_TYPE == 'POOL':
            response = requests.get('http://127.0.0.1:5000/random').text
            proxy = 'http://' + response
            return proxy
        elif PROXY_TYPE == 'TUNNEL':
            proxy_user = TUNNEL_PROXY_CONFIG.get('user')
            proxy_pass = TUNNEL_PROXY_CONFIG.get('pass')
            proxy_host = TUNNEL_PROXY_CONFIG.get('host')
            proxy_port = TUNNEL_PROXY_CONFIG.get('port')
            proxy = f'http://{proxy_user}:{proxy_pass}@{proxy_host}:{proxy_port}'
            return proxy
        elif PROXY_TYPE is None:
            return None
        else:
            raise SettingError('The PROXY_TYPE setting must POOL or TUNNEL or None')
        

if __name__ == '__main__':
    a = Downloader()
    print(a.get_proxy())

